import pandas as pd
def check_na(df, x_c, y_c, y_info={"y": 1, "n": 0}):
    rst0 = {}
    n = len(df)
    for k, v in y_info.items():
        s0 = df.loc[df[y_c] == v, :]
        k0 = y_c + "_" + k
        rst0[k0] = len(s0)
        rst0[k0 + "_r"] = len(s0) / n
    missing = df.isna()
    rst1 = {}
    x0 = x_c + [y_c]
    for i in x0:
        n0 = missing[i].sum()
        n1 = n - n0
        if n0 > 0:
            rst1[i] = {"n": n, "n0": n1, "na_n": n0, "na_n_r": 1000 * n0 / n}
    rst = pd.DataFrame(rst1).T
    if not rst.empty:
        rst.sort_values(by="na_n", inplace=True)
        isfirst = False
        for i in rst.index:
            if isfirst:
                filter = filter | df[i].isna()
            else:
                filter = df[i].isna()
                isfirst = True
            s0 = df.loc[filter, :]
            n2 = len(s0)
            rst1[i]["累计剔除个数"] = n2
            rst1[i]["累计剔除比"] = 1000 * n2 / n
            for k, v in y_info.items():
                s1 = s0.loc[s0[y_c] == v, :]
                k0 = "累计剔除" + y_c + "_" + k
                rst1[i][k0] = len(s1)
                rst1[i][k0 + "_r"] = len(s1) / n2
            for k, v in rst0.items():
                rst1[i][k] = v
        rst = pd.DataFrame(rst1).T
        rst.sort_values(by="na_n", inplace=True)
    return rst
def check_duplicate(df,x_c, y_c, y_info={"y": 1, "n": 0},isdrop=False):
    keys=None
    if x_c and y_c:
        keys=x_c+[y_c]
    if keys:
        duplicate_count = df.duplicated(subset=keys).sum()
        if isdrop:
            df.drop_duplicates(subset=keys,inplace=True)
    else:
        duplicate_count = df.duplicated().sum()
        df.drop_duplicates(inplace=True)
    return duplicate_count
def check_outlier(df,x_c, y_c, y_info={"y": 1, "n": 0}):
    rst={"3sigma":{},"boxplot":{}}
    desc = df.describe()
    # 找到异常值（超过3倍标准差的值）
    keys = df.columns
    if x_c:
        keys = x_c+[y_c]
    for col in keys:
        if desc[col]['count'] > 1:
            mean = desc[col]['mean']
            std = desc[col]['std']
            arg0=mean - 3 * std
            outliers = df[df[col] < arg0]
            if not outliers.empty:
                if col not in rst["3sigma"].keys():
                    rst["3sigma"][col]={}
                rst["3sigma"][col]["left"]={"n":len(outliers),"data":outliers[col]}
            arg1 = mean + 3 * std
            outliers = df[df[col] > arg1]
            if not outliers.empty:
                if col not in rst["3sigma"].keys():
                    rst["3sigma"][col] = {}
                rst["3sigma"][col]["right"] = {"n": len(outliers), "data": outliers[col]}
            IQR=desc[col]['75%']-desc[col]['25%']
            lower, upper = desc[col]['25%'] - 1.5 * IQR, desc[col]['75%'] + 1.5 * IQR
            outliers = df[df[col] < lower]
            if not outliers.empty:
                if col not in rst["boxplot"].keys():
                    rst["boxplot"][col]={}
                rst["boxplot"][col]["left"]={"n":len(outliers),"data":outliers[col]}
            outliers = df[df[col] > upper]
            if not outliers.empty:
                if col not in rst["boxplot"].keys():
                    rst["boxplot"][col] = {}
                rst["boxplot"][col]["boxplot"] = {"n": len(outliers), "data": outliers[col]}
    if y_info:
        s0=set(y_info.values())
        s1=set(df[y_c].to_list())
        s2=s1-s0
        if s2:
            rst[y_c]={}
            for i in s2:
                s0=df.loc[df[y_c]==i]
                rst[y_c][i]={"n":len(s0),"data":s0[i]}
    return rst